<?php
// Copyright 2020 Catchpoint Systems Inc.
// Use of this source code is governed by the Polyform Shield 1.0.0 license that can be
// found in the LICENSE.md file.
if(extension_loaded('newrelic')) {
  newrelic_add_custom_tracer('ProcessRun');
  newrelic_add_custom_tracer('loadAllPageData');
  newrelic_add_custom_tracer('getRequestsForStep');
  newrelic_add_custom_tracer('LockTest');
  newrelic_add_custom_tracer('UpdateTester');
  newrelic_add_custom_tracer('GetVisualProgressForStep');
  newrelic_add_custom_tracer('GetDevToolsCPUTimeForStep');
  newrelic_add_custom_tracer('GetDevToolsRequestsForStep');
  newrelic_add_custom_tracer('loadUserTimingData');
  newrelic_add_custom_tracer('GetVisualProgress');
  newrelic_add_custom_tracer('DevToolsGetConsoleLog');
  newrelic_add_custom_tracer('SecureDir');
  newrelic_add_custom_tracer('loadPageRunData');
  newrelic_add_custom_tracer('loadPageStepData');
  newrelic_add_custom_tracer('ParseUserTiming');
  newrelic_add_custom_tracer('CalculateTimeToInteractive');

}

chdir('..');
//$debug = true;
require_once('common.inc');
require_once('archive.inc');
require_once 'page_data.inc';
require_once('object_detail.inc');
require_once('harTiming.inc');
require_once('video.inc');
require_once('breakdown.inc');
require_once('devtools.inc.php');
require_once('./video/visualProgress.inc.php');
require_once __DIR__ . '/../include/ResultProcessing.php';

if (!isset($included)) {
  error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
  header('Content-type: text/plain');
  header("Cache-Control: no-cache, must-revalidate");
  header("Expires: Sat, 26 Jul 1997 05:00:00 GMT");
}
set_time_limit(3600);
ignore_user_abort(true);

$key  = isset($_REQUEST['key']) ? $_REQUEST['key'] : null;
$id   = isset($_REQUEST['id']) ? $_REQUEST['id'] : null;

if(extension_loaded('newrelic')) {
  newrelic_add_custom_parameter('test', $id);
}

$workdone_start = microtime(true);

//logmsg(json_encode($_REQUEST), './work/workdone.log', true);

// The following params have a default value:
$done = arrayLookupWithDefault('done', $_REQUEST, false);
$cpu = arrayLookupWithDefault('cpu', $_REQUEST, 0);
$pc = array_key_exists('pc', $_REQUEST) ? $_REQUEST['pc'] : '';
$ec2 = array_key_exists('ec2', $_REQUEST) ? $_REQUEST['ec2'] : '';
$tester = null;
if (strlen($ec2))
  $tester = $ec2;
elseif (strlen($pc))
  $tester = $pc . '-' . trim($_SERVER['REMOTE_ADDR']);
else
  $tester = trim($_SERVER['REMOTE_ADDR']);

// Sometimes we need to make changes to the way the client and server
// communicate, without updating both at the same time.  The following
// params can be set by a client to declare that they support a new
// behavior.  Once all clients are updated, they should be removed.

// Should zipped HAR uploads be flattened?  The main user is the mobitest iOS
// agent.  The agent will be updated to always set this, but until we can
// re-image all agents, we need to support the old behavior for a while.
$flattenUploadedZippedHar =
    arrayLookupWithDefault('flattenZippedHar', $_REQUEST, false);

// The following params are set by the android agents (blaze and WebpageTest).
// TODO(skerner): POST params are not saved to disk directly, so it is hard to
// see what the agent uploaded after the fact.  Consider writing them to a
// file that gets uploaded.
$runNumber     = arrayLookupWithDefault('_runNumber',     $_REQUEST, null);
$runNumber     = arrayLookupWithDefault('run',            $_REQUEST, $runNumber);
$runIndex      = arrayLookupWithDefault('index',          $_REQUEST, null);
$cacheWarmed   = arrayLookupWithDefault('_cacheWarmed',   $_REQUEST, null);
$cacheWarmed   = arrayLookupWithDefault('cached',         $_REQUEST, $cacheWarmed);
$docComplete   = arrayLookupWithDefault('_docComplete',   $_REQUEST, null);
$onFullyLoaded = arrayLookupWithDefault('_onFullyLoaded', $_REQUEST, null);
$onRender      = arrayLookupWithDefault('_onRender',      $_REQUEST, null);
$urlUnderTest  = arrayLookupWithDefault('_urlUnderTest',  $_REQUEST, null);

$testInfo_dirty = false;

if (ValidateTestId($id)) {
  $testPath = './' . GetTestPath($id);
  $location = null;
  if (isset($_REQUEST['location'])) {
    $location = $_REQUEST['location'];
  } else {
    $testInfo = GetTestInfo($id);
    if (isset($testInfo['location']))
      $location = $testInfo['location'];
  }
  if (isset($location)) {
    $locKey = GetLocationKey($location);
    if ((!strlen($locKey) || !strcmp($key, $locKey)) || !strcmp($_SERVER['REMOTE_ADDR'], "127.0.0.1")) {
      // Extract the uploaded data
      if (isset($_FILES['file']['tmp_name'])) {
        ExtractZipFile($_FILES['file']['tmp_name'], $testPath);
        CompressTextFiles($testPath);
      }

      $testErrorStr = '';
      $errorStr = '';
      if (array_key_exists('testerror', $_REQUEST) && strlen($_REQUEST['testerror']))
        $testErrorStr = ', Test Error: "' . $_REQUEST['testerror'] . '"';
      if (array_key_exists('error', $_REQUEST) && strlen($_REQUEST['error']))
        $errorStr = ', Test Run Error: "' . $_REQUEST['error'] . '"';
      $testLock = LockTest($id);
      logTestMsg($id, "Test Run Complete. Run: $runNumber, Cached: $cacheWarmed, Done: $done, Tester: $tester$testErrorStr$errorStr");
      if (!isset($testLock))
        logTestMsg($id, "Failed to lock test");
      $testInfo = GetTestInfo($id);

      // Figure out the path to the results.
      $ini = parse_ini_file("$testPath/testinfo.ini");
      $time = time();
      $testInfo['last_updated'] = $time;
      // Allow for the test agents to indicate that they are including a
      // trace-based timeline (mostly for the mobile agents that always include it)
      if (isset($_REQUEST['timeline']) && $_REQUEST['timeline'])
        $testInfo['timeline'] = 1;
      $testInfo_dirty = true;

      if (!strlen($tester) && array_key_exists('tester', $testInfo) && strlen($testInfo['tester']))
        $tester = $testInfo['tester'];

      $medianMetric = GetSetting('medianMetric', 'loadTime');
      if (isset($testInfo['medianMetric']))
        $medianMetric = $testInfo['medianMetric'];
    
      // keep track of any overall or run-specific errors reported by the agent
      if (array_key_exists('testerror', $_REQUEST) && strlen($_REQUEST['testerror'])) {
        $testInfo['test_error'] = $_REQUEST['testerror'];
        $testInfo_dirty = true;
      }
      if (isset($runNumber) &&
          isset($cacheWarmed) &&
          array_key_exists('error', $_REQUEST) &&
          strlen($_REQUEST['error'])) {
        if (!array_key_exists('errors', $testInfo))
          $testInfo['errors'] = array();
        if (!array_key_exists($runNumber, $testInfo['errors']))
          $testInfo['errors'][$runNumber] = array();
        $testInfo['errors'][$runNumber][$cacheWarmed] = $_REQUEST['error'];
        $testInfo_dirty = true;
      }

      // Do any post-processing on this individual run
      ProcessRun();

      if (strlen($location) && strlen($tester)) {
        $testerInfo = array();
        $testerInfo['ip'] = $_SERVER['REMOTE_ADDR'];
        if (!isset($testerError))
          $testerError = false;
        if (array_key_exists('testerror', $_REQUEST) && strlen($_REQUEST['testerror']))
          $testerError = $_REQUEST['testerror'];
        elseif (array_key_exists('error', $_REQUEST) && strlen($_REQUEST['error']))
          $testerError = $_REQUEST['error'];
        // clear the rebooted flag on the first successful test
        $rebooted = null;
        if ($testerError === false)
          $rebooted = false;
        UpdateTester($location, $tester, $testerInfo, $cpu, $testerError, $rebooted);
      }

      // see if the test is complete
      $all_done = false;
      if ($done) {
        $all_done = true;
        if (isset($runNumber)) {
          $all_done = false;
          $runnum = intval($runNumber);
          if ($runnum) {
            touch("$testPath/run.complete.$runnum");
            $files = glob("$testPath/run.complete.*");
            if (isset($files) && is_array($files)) {
              $done_count = count($files);
              logTestMsg($id, "$done_count of {$testInfo['runs']} tests complete");
              if ($done_count >= $testInfo['runs']) {
                logTestMsg($id, 'All done');
                $all_done = true;
              }
            }
          }
        }
        if ($all_done) {
          // Mark the test as done and save it out so that we can load the page data
          $testInfo['completed'] = $time;
          SaveTestInfo($id, $testInfo);
          $testInfo_dirty = false;

          // delete any .test files
          $files = scandir($testPath);
          foreach ($files as $file)
            if (preg_match('/.*\.test$/', $file))
              unlink("$testPath/$file");
          if (array_key_exists('job_file', $testInfo) && is_file($testInfo['job_file']))
            unlink($testInfo['job_file']);

          $perTestTime = 0;
          $testCount = 0;

          // do pre-complete post-processing
          MoveVideoFiles($testPath);
          WptHookPostProcessResults(__DIR__ . '/../' . $testPath);

          if (!isset($pageData))
            $pageData = loadAllPageData($testPath);
          $medianRun = GetMedianRun($pageData, 0, $medianMetric);
          $testInfo['medianRun'] = $medianRun;
          $testInfo_dirty = true;

          // delete all of the videos except for the median run?
          if( array_key_exists('median_video', $ini) && $ini['median_video'])
            KeepVideoForRun($testPath, $medianRun);

          $test = file_get_contents("$testPath/testinfo.ini");
          $now = gmdate("m/d/y G:i:s", $time);

          // update the completion time if it isn't already set
          if (!strpos($test, 'completeTime')) {
            $complete = "[test]\r\ncompleteTime=$now";
            if($medianRun)
              $complete .= "\r\nmedianRun=$medianRun";
            $out = str_replace('[test]', $complete, $test);
            file_put_contents("$testPath/testinfo.ini", $out);
          }
        }
      }

      if ($testInfo_dirty)
        SaveTestInfo($id, $testInfo);

      SecureDir($testPath);
      logTestMsg($id, "Done Processing. Run: $runNumber, Cached: $cacheWarmed, Done: $done, Tester: $tester$testErrorStr$errorStr");
      UnlockTest($testLock);
      /*************************************************************************
      * Do No modify TestInfo after this point
      **************************************************************************/

      // do any post-processing when the full test is complete that doesn't rely on testinfo
      if ($all_done) {
        logTestMsg($id, "Test Complete");

        touch("$testPath/test.complete");
        @unlink("$testPath/test.running");
        @unlink("$testPath/test.waiting");
        @unlink("$testPath/test.scheduled");

        // Cleanup the files marking each run
        $files = glob("$testPath/run.complete.*");
        if (isset($files) && is_array($files)) {
          foreach ($files as $file) {
            if (file_exists($file)) {
              unlink($file);
            }
          }
        }

        // send an async request to the post-processing code so we don't block
        SendAsyncRequest("/work/postprocess.php?test=$id");
      }
    } else {
      logMsg("location key incorrect\n");
    }
  }
}

$workdone_end = microtime(true);

function ProcessRun() {
  global $runNumber, $cacheWarmed, $testPath, $id, $testerError;
  if (isset($runNumber) && isset($cacheWarmed)) {
    $resultProcessing = new ResultProcessing($testPath, $id, $runNumber, $cacheWarmed);
    $testerError = $resultProcessing->postProcessRun();
  }
}

/**
* Delete all of the video files except for the median run
*
* @param mixed $id
*/
function KeepVideoForRun($testPath, $run)
{
  if ($run && !GetSetting('keep_all_video')) {
    $dir = opendir($testPath);
    if ($dir) {
      while($file = readdir($dir)) {
        $path = $testPath  . "/$file/";
        if( is_dir($path) && !strncmp($file, 'video_', 6) && $file != "video_$run" )
          delTree("$path/");
        elseif (is_file("$testPath/$file")) {
          if (preg_match('/^([0-9]+(_Cached)?)[_\.]/', $file, $matches) && count($matches) > 1) {
            $match_run = $matches[1];
            if (strcmp("$run", $match_run) &&
                (strpos($file, '_bodies.zip') ||
                 strpos($file, '.cap') ||
                 strpos($file, '_trace.json') ||
                 strpos($file, '_netlog.txt') ||
                 strpos($file, '_doc.') ||
                 strpos($file, '_render.'))) {
              unlink("$testPath/$file");
            }
          }
        }
      }

      closedir($dir);
    }
  }
}

/**
 * Remove sensitive data fields from HTTP headers (cookies and HTTP Auth)
 *
 */
function RemoveSensitiveHeaders($file) {
    $patterns = array('/(cookie:[ ]*)([^\r\n]*)/i','/(authenticate:[ ]*)([^\r\n]*)/i');
    $data = gz_file_get_contents($file);
    $data = preg_replace($patterns, '\1XXXXXX', $data);
    gz_file_put_contents($file, $data);
}

/**
* Reset the state of the given test directory (delete all the results)
*
* @param mixed $testDir
*/
function ResetTestDir($testPath) {
    $files = scandir($testPath);
    foreach ($files as $file) {
        if ($file != '.' && $file != '..' && strncasecmp($file, 'test', 4)) {
            if (is_file("$testPath/$file"))
                unlink("$testPath/$file");
            elseif (is_dir("$testPath/$file"))
                delTree("$testPath/$file");
        }
    }
}

function CompressTextFiles($testPath) {
  global $ini;
  $f = scandir($testPath);
  foreach( $f as $textFile ) {
    if ($textFile != 'test.log') {
      logMsg("Checking $textFile\n");
      if( is_file("$testPath/$textFile") ) {
        $parts = pathinfo($textFile);
        $ext = $parts['extension'];
        if( !strcasecmp( $ext, 'txt') ||
            !strcasecmp( $ext, 'json') ||
            !strcasecmp( $ext, 'log') ||
            !strcasecmp( $ext, 'csv') ) {
          if (strpos($textFile, '_optimization'))
            unlink("$testPath/$textFile");
          elseif (gz_compress("$testPath/$textFile"))
            unlink("$testPath/$textFile");
        }
        if (isset($ini) && is_array($ini) && isset($ini['sensitive']) && $ini['sensitive'] && strpos($textFile, '_report'))
          RemoveSensitiveHeaders($testPath . '/' . basename($textFile, '.gz'));
      }
    }
  }
}

function ExtractZipFile($file, $testPath) {
  global $id;
  $zipsize = filesize($file);
  logTestMsg($id, "Extracting $zipsize byte uploaded file '$file' to '$testPath'");
  $zip = new ZipArchive();
  if ($zip->open($file) === TRUE) {
    $valid = true;
    // Make sure all of the uploaded files are appropriate
    for ($i=0; $i < $zip->numFiles; $i++) {
      $entry = $zip->getNameIndex($i);
      if (substr($entry, -1) == '/') continue; // skip directories
      $fileName = basename($entry);
      if (!validateUploadFileName($fileName))
        $valid = false;
    }
    if ($valid) {
      $extractPath = realpath($testPath);
      if ($extractPath !== false) {
        if (!$zip->extractTo($extractPath))
          logTestMsg($id, "Error extracting uploaded ZIP file '$file' to '$testPath'");
        $zip->close();
      }
    }
  } else {
    logTestMsg($id, "Error opening uploaded ZIP file '$file'");
  }
}
?>
